#!/bin/env pypy

# Train the word rank model from CORPUS and save it into 'output/word_rank.txt'
#
# Configure parameters in src/confs.py

import os
import sys
import subprocess
import datetime

CORPUS = '../data/brent.raw.txt'
# CORPUS = '../data/large.raw.txt'

ebv_iterations = 30

_verbose = False

def call(cmd):
    if _verbose:
        print '$ ' + cmd
    subprocess.check_call(cmd, shell = True)

def check_output(cmd):
    return subprocess.check_output(cmd, shell = True).strip()

def map_reduce(name, mapper, reducer, input_paths, output_path):
    if name is None:
        name = ''
    if type(input_paths) != list:
        input_paths = [input_paths]
    print 'Map Reduce Job:', name
    print '  Mapper:', mapper
    print '  Reducer:', reducer
    print '  Input:', input_paths
    print '  Output:', output_path
    if os.path.exists(output_path):
        print "'%s' already exists, SKIPPED" % output_path
    else:
        t1 = datetime.datetime.now()
        call('rm -f tmp1')
        for input_path in input_paths:
            print 'mapping from %s' % input_path
            call('%s < %s >> tmp1' % (mapper, input_path))

        print 'sorting..'
        call("bin/emsx tmp1 tmp2")

        print 'reducing to %s..' % output_path
        call('%s < tmp2 > %s' % (reducer, output_path))

        call('rm -f tmp1 tmp2')

        t2 = datetime.datetime.now()
        delta = t2 - t1
        print 'time elapsed:', delta.seconds, 'seconds'

def main():
    map_reduce('Build Mutual Infomation Model', 
               'pypy src/mutual_infomation_model_map.py', 
               'pypy src/mutual_infomation_model_reduce.py', 
               CORPUS, 
               'output/mi_model.txt')
    print "mutual infomation model built in 'output/mi_model.txt'\n"

    map_reduce('Retrieve Word Hypothesis and Counting', 
               'pypy src/retrieve_words_map.py', 
               'pypy src/retrieve_words_reduce.py', 
               CORPUS,
               'output/words_all.txt')
    print
    map_reduce('Filter Word Hypothesis', 
               'pypy src/filter_words_map.py', 
               'pypy src/filter_words_reduce.py', 
               'output/words_all.txt',
               'output/words_filtered.txt')
    print "word hypothesises retrieved in 'output/words_filtered.txt'\n"

    map_reduce('Build Link Structure First Stage', 
               'pypy src/build_links_first_map.py', 
               'cat', 
               CORPUS,
               'output/links_first.txt')
    print
    map_reduce('Build Link Structure Second Stage', 
               'cat', 
               'pypy src/build_links_second_reduce.py', 
               ['output/links_first.txt', 'output/words_filtered.txt'],
               'output/links_second.txt')
    print
    map_reduce('Build Link Structure Third Stage', 
               'cat', 
               'pypy src/build_links_third_reduce.py', 
               'output/links_second.txt',
               'output/links_third.txt')
    print
    map_reduce('Build Link Structure Fourth Stage', 
               'cat', 
               'pypy src/build_links_fourth_reduce.py', 
               'output/links_third.txt',
               'output/links.txt')
    print "link structure built in 'output/links.txt'\n"

    map_reduce('Exterior Boundary Values Initialization Counting', 
               'pypy src/ebv_init_count_map.py', 
               'pypy src/ebv_init_count_reduce.py', 
               'output/links.txt', 
               'output/ebv_init_count.txt')
    print "init count saved in 'output/ebv_init_count.txt'\n"

    for i in range(1, ebv_iterations + 1):
        map_reduce('Exterior Boundary Values Iteration %d' % i, 
                   'pypy src/ebv_iterate_map.py', 
                   'pypy src/ebv_iterate_reduce.py', 
                   'output/ebv_%d.txt' % (i - 1) if i > 1 else 'output/links.txt', 
                   'output/ebv_%d.txt' % i)
        print
    print "exterior boundary values saved in 'output/ebv_%d.txt'\n" % ebv_iterations

    map_reduce('Interior Boundary Values and Word Rank Calculation', 
               'pypy src/word_rank_map.py', 
               'cat',
               'output/ebv_%d.txt' % ebv_iterations, 
               'output/word_rank.txt')
    print "word rank saved in 'output/word_rank.txt'\n"

if __name__ == '__main__':
    if '-v' in sys.argv[1: ]:
        _verbose = True
    for arg in sys.argv[1: ]:
        if not arg.startswith('-'):
            ebv_iterations = int(arg)

    t1 = datetime.datetime.now()
    main()
    t2 = datetime.datetime.now()
    delta = t2 - t1
    print 'total run time:', delta.seconds, 'seconds'
